{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\n",
    "from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier\n",
    "from sklearn.metrics import r2_score, mean_squared_error, accuracy_score, f1_score, recall_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "from xgboost import XGBClassifier, XGBRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize(score_list):\n",
    "    train_score_list = [i['train_score'] for i in score_list]\n",
    "    valid_score_list = [i['valid_score'] for i in score_list]\n",
    "    plt.plot(train_score_list, label='train_score')\n",
    "    plt.plot(valid_score_list, label='valid_score')\n",
    "    plt.grid()\n",
    "    plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(537577, 11) (537577,)\n",
      "(483819, 11) (483819,)\n",
      "(53758, 11) (53758,)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./BlackFriday.csv')\n",
    "X = df.loc[:, df.columns != 'Purchase'].copy()\n",
    "y = df.loc[:, df.columns == 'Purchase'].copy()\n",
    "\n",
    "for c in ['Product_ID', 'Gender', 'Age', 'City_Category', 'Stay_In_Current_City_Years']:\n",
    "    X[c] = LabelEncoder().fit_transform(X[c])\n",
    "\n",
    "X = SimpleImputer().fit_transform(X)\n",
    "y = y.values.reshape(-1)\n",
    "\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "print(X.shape, y.shape)\n",
    "print(train_X.shape, train_y.shape)\n",
    "print(valid_X.shape, valid_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyGradientBoostingRegressor:\n",
    "    \n",
    "    def __init__(self, n_estimators=100, lr=0.1, max_depth=3, verbose=False):\n",
    "        self.n_estimators = n_estimators\n",
    "        self.lr = lr\n",
    "        self.max_depth = max_depth\n",
    "        self.verbose = verbose\n",
    "        \n",
    "        self.estimator_list = None\n",
    "        self.is_first = True\n",
    "        self.F = None\n",
    "        self.score_list = list()\n",
    "        \n",
    "    def fit(self, train_X, train_y):\n",
    "        self.estimator_list = list()\n",
    "        self.F = np.zeros_like(train_y, dtype=float)\n",
    "        \n",
    "        for i in range(1, self.n_estimators + 1):\n",
    "            # get negative gradients\n",
    "            neg_grads = train_y - self.F\n",
    "            base = DecisionTreeRegressor(max_depth=self.max_depth)\n",
    "            base.fit(train_X, neg_grads)\n",
    "            train_preds = base.predict(train_X)\n",
    "            self.estimator_list.append(base)\n",
    "            \n",
    "            if self.is_first:\n",
    "                self.F = train_preds\n",
    "                self.is_first = False\n",
    "            else:\n",
    "                self.F += self.lr * train_preds\n",
    "                \n",
    "            train_score = r2_score(train_y, self.F)\n",
    "            valid_preds = self.predict(valid_X)\n",
    "            valid_score = r2_score(valid_y, valid_preds)\n",
    "            iter_score = dict(iter=i, train_score=train_score, valid_score=valid_score)\n",
    "            self.score_list.append(iter_score)\n",
    "            if self.verbose:\n",
    "                print(iter_score)\n",
    "                \n",
    "    def predict(self, X):\n",
    "        F = np.zeros_like(len(X), dtype=float)\n",
    "        is_first = True\n",
    "        for base in self.estimator_list:\n",
    "            preds = base.predict(X)\n",
    "            if is_first:\n",
    "                F = preds\n",
    "                is_first = False\n",
    "            else:\n",
    "                F += self.lr * preds\n",
    "        return F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor train_score: 0.7106 valid_score: 0.7037\n",
      "MyGradientBoostingRegressor train_score: 0.7121 valid_score: 0.7054\n"
     ]
    }
   ],
   "source": [
    "model = GradientBoostingRegressor(n_estimators=300, max_depth=5)\n",
    "model.fit(train_X, train_y)\n",
    "print('GradientBoostingRegressor train_score: {:.4f} valid_score: {:.4f}'.format(\n",
    "    r2_score(train_y, model.predict(train_X)), r2_score(valid_y, model.predict(valid_X))))\n",
    "\n",
    "model = MyGradientBoostingRegressor(n_estimators=300, max_depth=5)\n",
    "model.fit(train_X, train_y)\n",
    "print('MyGradientBoostingRegressor train_score: {:.4f} valid_score: {:.4f}'.format(\n",
    "    r2_score(train_y, model.predict(train_X)), r2_score(valid_y, model.predict(valid_X))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8VPWd//HXN5OZSSbXSUJCIIEQwAuI3CJ4K4a2KOpWsWrVarW2u7pbL9321666dtWq29L2sdtHfSyu1ZbH+rNWVNoi/oparUZWAeVS0HC/kwRC7slM5j7z/f1xTsIQEgi5zWTm83w85jEzZ86Z+Xwz4Z3D93zP9yitNUIIIZJDSqwLEEIIMXIk9IUQIolI6AshRBKR0BdCiCQioS+EEElEQl8IIZKIhL4QQiQRCX0hhEgiEvpCCJFEUmNdQE8FBQW6rKxswNt3dnaSkZExdAXFUKK0JVHaAdKWeCVtgc2bNzdprcecab24C/2ysjI2bdo04O2rqqqorKwcuoJiKFHakijtAGlLvJK2gFLqcH/Wk+4dIYRIIhL6QgiRRCT0hRAiiUjoCyFEEpHQF0KIJCKhL4QQSURCXwghkkjcjdMXQohEFo5omjv9NLkCNLn93Te3P8zY7DTGDfPnS+gLIcQAaK1x+0O0eYLGzRugpTNAsztAqydAc2eA1s4AnYEw/mCYNk+QJrefFk+Avi5NPntCLt+dNrx1S+gLIQRGiLd5gjS6/TR0+Glw+Whw+Wl2+/EGw/iCEY53+DjW7qO1M0CbN0g40nt6pyhwOmw4M2xk2lOxpaYwId/BnIlOxmTaKMiyU5DZdTOeZ9lTUUpRVVU1rO2U0BdCJKxwRNPmCdDqCdLqCdDk8tPo9nffN7oCJ567/ATCkVPeI82agsOWij01hcIsO5PHZJA/KY/cdCtOh41ch5Vc8z4vw0aew0ZOupWUFBWDFp+ZhL4QYlTxBsI0uf00dxoh3tzpZ+P+AB+6ttPsNvrJm90Bmjv9tHQG6G1nXCnIz7BRkGlnjBnkhVlpFGYZzwuz7BRmG88z7IkVk4nVGiHEqBMIRfAEQjS5A9S1eWl0+WnzGP3jrZ4ATe4AzW5/931nINzr+2QeqSU/00Z+ho2JUV0peRlGN4vTcSLk8zJsWOJ0T3y4SegLIYaF1hqXP0Rrp3FQs6bFw6EmD8ddPppcxoiVY+0+6jt8vR7YTE1R5DqMEC/IsjGrNJeCTDv5mTbGmPf5Zp/49i2fcNWXFo58I4dCOAgBNwR9xn9BhpmEvhBiQDyBELWtXmpaPBxp8VBvBnh9u687zAOhk/vIu7pV8jOM0L5kcj6lTgc56UZ/+HhnOoVZdpwZtu4Dm/2xzxLjvfZQAHxt4GkB11For4VApxHo4QBEQuDrAG8LeJrNW4tx87efeJ+Si2DKj4a1VAl9IQRgdLMYBzf9NHQYI1caXf7u+ya3nw5vkA5fkA5f6JRAt6WmMDY7jbHZacwqzaU4J42CTKMrpSvQJ+Q5SLNaYtTCfgp4oLMR3A3QcgCCZngHveA+Dh11Rlh728DbatyCnWd+X2sGOPLB4TTunZPM5/lgzwJrGmQVw7HhbV6/Ql8ptRj4FWABfqO1Xtrj9V8CXf+3cgCFWutc87W7gK4/XU9rrV8cisKFEGen3RukttVDTYuxd/5RtZ8X9m3oDvY2T/CUbZSCPIeNMeYBzhJnOllpVrLTU8lOs1JiBnlpnoP8DFu/98xHRChg7FkHOiHoAb87ai+7yQjuzqaTl3U2Gev2xeqA7HGQMQZySmDsDEh3mrdc4z6rGHLGgz0bLFaw2CDFCin9nADhWNWQNL8vZwx9pZQFWAYsAmqBjUqp1VrrHV3raK2/F7X+A8Bs83Ee8DhQAWhgs7lt65C2QghxUndL131N64nHHb7QSetn2WByUZhJBRnMn5TfPWrFuE9jTJbRBWO1xMlsLeEQ+NqNbhRvG/hawdMK7nrK92+Clt+Dq97YQ3cfNwL/dLr3vPMgowAKphph7sg37jPGQF65sRdusUGqDWyZI9LvPpz6s6c/D9intT4AoJRaAVwP7Ohj/dswgh7gKuBdrXWLue27wGLglcEULUSycfmCHG72cLjZw6HmTg41dZqn7odw+UI0uowhjNHSrCmUOB2UOtOZO9FJiTOdUqexV17qdPC3Tz+msvKyGLUICPnBdcwI6qDH7A+vN5a5G8w970Yj2H1txsHOPpQoK7iKIXMs5E+GiZdC1lgj0G2Zxh66LcMI964uFWv6CDY2fvQn9McDNVHPa4H5va2olJoITALeP82248++TCESWySiOe7ycaTZw+EWD0eajYOjxuNOWnt0vRjjyO1k2a2U5jmYVZpLaZ7DCHYz1AsyY9TdEgkbYe06Bh3HTgS766hx37Wsrz3x1HTILDQCOns8FM0wuk7Sck/cp+WY3Sl5kFXE2vVbqFw4SkfvjLChPpB7K7BSa937QNo+KKXuAe4BKCoqGtRpyG63e9hPYx4pidKWRGkHDKwtWmva/JoGj8Yd1HiCGk8ImrwRGjyaRk+ERq8mGHVcNEVBfpqi0KGYmZ9CYamVMekpFDoURY4U0lIVEDZvXVqgDTraYPtQtkVrLGEP1qALS9iHJezFEvZi97diC7Rg9zeb98bNFmhFcfJBXk0KAVsOfns+AVsefuc8AkX5+O15BGx5hC1pBK2ZBGx5hFIz+u5C8Zo3AHzAUeAo7s7OpP4dOxv9Cf06oDTqeYm5rDe3Avf12Layx7ZVPTfSWj8PPA9QUVGhB3NV+4FeST4eJUpbEqUdcPq2eAIhDjZ1cqDRvDW5zcfuXk8oyrBZmJCfwYVlDibkO5iQ52CieT8uN314+9K15uN33+CyKVnGXnlngzlipdHsUmmCzq4Dns3GsMO+pOUaBzcLSyFrvtGtkjXWWJY1FrKKURmF2C2p2IepOcnyOzYU+hP6G4GpSqlJGCF+K/D1nisppc4DnMD6qMXvAD9RSjnN51cCjwyqYiFiyB/WbKtp40CTm3ZPkCZ3gJ3HOthV76KurXsXFKVgXE465WMyuLmilPIxGUwqyCAvw0Z2mtW4pfd/HPqAhING33jbEWjcBc37oOUgtB6C1kNcFuyEdT22sWVBRv6J0SnjZoKjwOhqSc8De6bRN27LhMwiI9STtG98tDpj6GutQ0qp+zEC3AIs11pvV0o9CWzSWq82V70VWKH1iXPrtNYtSqmnMP5wADzZdVBXiHjV7Paz+7iLY20+jrV7Odru41iblyMtHg40etDvfty9boqCKYWZVJQ5uXVMKeVjMrsDftjGo/tdRpC3Hjb2xAOd5kHQY8aola4+9M4mjEFzptR0cJYZt/Ir2NsUYupFXzT7z83RKhLgCa9fffpa6zXAmh7LHuvx/Ik+tl0OLB9gfUIMm+MdPvYed9Pg8nGkxcOuYy6qj7ZT2+o9ab28DBtjs9OYPCaTC7IDXH3JDKYUZpGXYSMrLXXou2GCPmivMUK97ZB5fwTaDhuPez0AqozwzhoLWeNg3BxjvHhWEeSUwphzIbvkpLHidVVVTD23cmhrF3FPzsgVCc8fCtPQ4ed4h49d9S42H25l0+EWalpO7o4py89gZkkud14ykWnFOYx3plOck3bSHntVVRWVFxQPvJhIxDjt3tNinKrfFeTRoe6uP3kbiw1yJ0DuRBg3+8Rj50TIKDS6W+zZYJF/zuLM5LdEJIxIRHOwuZNPD7awbn8z+xrcHO/w0dJj/HpBpp2KiU7uuqSM6eNyKMq2MzYnDYdtCP45aG0G+hGo22KEeXutcep+ey10HIWeg9uUxTiDM3ciTP2ycd8V6rkTjb7z/p7NKcQZSOiLUUVrTaPb3z2VwJEWDzXmePZdxzq6zzotzLIzY3wOsyfkMjY7jaJsO0XZaUwqyGBCnmNwB1CD3hP95u7j0LQXmvYYB0sb90DYf2LdFKsR6NklxglD2eONvvN0pxn0E4zXZC9djBD5TRNxq8ntZ+PBFg41ezjW7qW21cvmw620e08+Uako286EPAfXXjiO2aW5zJnoZPKYjIEHu6/DCPG2I0bfensttNVAew2XNR+Eql4m18ophYJzYNIVxuPsYiieZTyWvXQRRyT0RVzwBcNsOdLKJwda2H60ne1HOzjW7ut+PTstlXG56Vw5rYgLxud0T/JV4kwf2CiZSMSYQbH1oDEuve0IHK+G+mpjWbS0HCO8c0ppsExg/PkVxsHSzELjlldu9KsLMQpI6IuYcPtDfLCrgb8daaNqdwOHWzyEI5oUBZPHZDJ/Uh7Tx+Uwt8zJOUVZZA7kknVag7/D6EfvbDQm62rYCQfXGv3tJ02Hq4w5W4ovhNl3QNF0o+slpxTSsrvX2ltVxfgvVA66/ULEioS+GDZaa2pbvexsDlP/6RH2HHfT6PZzrM3L53Xt+EMRbJYULp2SzzUzipk9IZeLJuWRnWYd2Af62qHmUyPUD31kdNGcMkmXMqbDnX2HEfAF5xgnHmWOBZtj0G0WIt5J6Isho7Vmf2Mn6/Y3sW5fM58cbD4xUdjGz0m3WijKNqbtvX3+RK6eMZZZpbn9H+ceCRt97M37jLNKWw5C835j3HpbjTGhFxgHT0sqjGDPHm9MB5BZaEwXkFNizLwoRJKS0BeDEgpH2N/Yyasba/jz50c53mGMXBmfm86Xzy9i9gQnbbV7ubbyYkqdDlL6ezFqT4ux117zCTTuNoP+4MlzwFjsRpdMRgFM+gKMOc8Yx146X/baheiDhL44a9V17bxVfYyP9zWztaYNAKtF8cXzCqk8t5DLJhdQmpfePXqmynuAifmnOdDpboCjW+HoFjiy3uh3dx83XktJhfwpxgUuzl1sPM6bbBw8lfHrQpw1CX1xRsc7fPxlez0bDrSw+XAr9R0+UlMU08dlc//CKRTnprF4+ljyM/sxh6LWRhdN7UbY/ifjgGpH16StCoougCmLjGkDxs+F8XNkPhghhpCEvjiF1prdx118vK+ZdfuaWLu3kWBYMy4njfnleVSU5XH9rHGnP+Dq6zAOqDbs4Jzdn8Lh/zQOrPraT1yDNKsYyi43xrOPmwVjLzxppIwQYuhJ6AtC4QiHmj1sOdzKx/ub+HhfM01uo29+UkEG37i4jK/Pn3D6E57CQTj0v7DzTSPsWw9DxDiIW2DNgTGTYfIXjTHv+VOMPfqSCkgZppkohRC9ktBPUv5QmL9sP86KjUfYeLCVQNi40lFBpp3Lp+Rz6ZQCLptSwPjcPrpWAp1wZIPRB99eC7vfMq5janVA+UI4/zqYugiKZ7Fu3acJc4ELIUY7Cf0kEY5o1u9v5tODzRxu8fC/e5to6QxQ4kznrksnct7YbGaU5DC1MLP3vflQAOo/M/rgd/8ZDlSZLyjjWqXnLIZp1xl789IHL0TcktBPYJsOtbDxUCuHmzt5f1cDDS4/lhRFcU4a8yflcdu8CVw+paDvYZRBn3HAdcuLsPP/QcicijhrHCz4oTFn++QvgjVt5BolhBgUCf0EE45o3qo+xlvV9fz5s2MA5GfYmDPRyQ2zx7Pw3ELSbX30o/s6jD34xl3G/eF1gDYujTfr6zBpAZRcZJzsNJyX+RNCDBsJ/QRR2+ph1d/qWLm5lkPNHnLSrdy7oJz7vziFrL5G2XhaYNsrRt98y0Fo3AkRY2pinJNgwQ+geCaUV4I9a6SaIoQYRhL6o5jbH2LN58f445ZaNhwwLqE3f1Ie/7L4PBZPH3tqt42vA+o/N8bF7/0L7HjDOMM1b7IxombqIpjyZWNUTWo/xtwLIUYdCf1Rpt0TZPORFlZvPcrb2+vxBSNMKsjg/yw6hyWzx1OaFzX9QOthOLbNCPkDVbD//RPTGNhzYO43jVvR9Bi0RAgRCxL6o0Bdm5e3q+t5Z3s9mw61ENGQk27lprklfHVOCbNLc08ecVO7CXasgg3PdY+VJ6cU5t1jdNXkToTcUhllI0QSktCPU6FwhO1NYd7+w2e8tqmGiIZzi7K4f+EU5pfnU1HmxJ4adUA25Df25j/+FRz+GFBwwVfh0gfNS/QVyMFXIYSEfrypbfXw2sYaXttUS32HD6ulljsvKeObl5ZRVtBj0jK/C3a/DTvfgH3vGxcFyRoHi5fCzNuM8fNCCBFFQj9ObDjQzBOrt7Or3oVSsGDqGG4s19z31UocNvNr0hoadhgzUu5eA3vfNS7CnTkWZt5inCBVXikHYYUQfZLQjyFPIMQbW4/y5rajrD/QTFl+Bg8tPo+vzCymxOmgqqoKh9UCe9+DXW/CnnfAZYy9J3OscRB2+g3G/PEyxbAQoh8k9GOg0eXn1Y1H+O1HB2n1BJlUkMEDC6dwzxWTT1wLtnEPEw+9Brv+zZj+wJYJkxcae/Pj50LBuRL0QoizJqE/ghpdfv7z3T28tqmGcETzxfMKuW/hZOZMcBqjb45vh31/heqVcGwbZV3Xc73+WZhxk3TbCCEGTUJ/BOxrcPP7T47w6sYj+EMR7pg/gTsunsjUoizwu2H/X2HLS8YwSzAu+XfVT1nfMZZLr/pqbIsXQiQUCf1h5PaHePLN7by2qRarRXH1BcX885enUj4mE+qr4U//BZ+vNMbSW+xQ+QjMut0YQw8Eqqpi2wAhRMKR0B8GkYjmLzuO85M1O6lt9XDvgnL+YUE5BekpsG0FrPw1HP8crBlQcTece7V5Me/TXEdWCCGGgIT+EAqEIqzaWsevP9zP/sZOyvIdvHrvJVw0RsO2F+DT56HtsHFZwKt+Yoyld+TFumwhRBKR0B8C3kCYlz85zG/+9yD1HT6mFWfzzG2zuWaqg9QPl8JLvzXmvCmdD9f8AqZeKWfHCiFiQkJ/kLyBMHcu/4SNh1q5uDyPn910IQvKMlC718DzT0LbEZh9B1z8HSiaFutyhRBJTkJ/gLTWVO1u5N/X7ORAo5tf3TqL60u8sOGn8MeV4GuHvHL41jswYX6syxVCCEBCf0B21Xfw2KrtfHqohbJ8By/fNJZLDj4Fq1+HlFTjouCzb4eyBXIClRAirkjon6Wdxzr4+gsbsKQonlpyAbeUh7D97jrwtkLFt42rTWUWxrpMIYTolYR+Px1u7uRXf93Lqr/VkZdh5093TaF0x/Pw6xcgNQ2+9TYUXxjrMoUQ4rQk9Pth+UcHWfr2LlIUfPvySfyz5XUyln8N0DDz67DwEcgpiXWZQghxRhL6p+ELhvnle3v49YcH+PL5RSz9spOCrc/Cht/AjJvhioegYGqsyxRCiH7rV+grpRYDvwIswG+01kt7WedrwBOABrZprb9uLg8Dn5urHdFaXzcEdQ+7A41u7n1pM3sb3Nwz087Dab8l5bevGXPaV3zbGG+fYjnzGwkhRBw5Y+grpSzAMmARUAtsVEqt1lrviFpnKvAIcJnWulUpFX0k06u1njXEdQ+rw82dfO3XG4hozes35nHRh980DtRe9A9wyXcgd0KsSxRCiAHpz57+PGCf1voAgFJqBXA9sCNqnX8AlmmtWwG01g1DXehIaXT5uXP5p+hwkPfmbiDv7WchLQfuqZKTq4QQo57SWp9+BaVuAhZrrf/efP4NYL7W+v6odVYBe4DLMLqAntBav22+FgK2AiFgqdZ6VS+fcQ9wD0BRUdHcFStWDLhBbrebzMzMAW3rCmh+vtFHu8fLW85fMt79GccLF3Cg/C78aQUDrmmgBtOWeJIo7QBpS7yStsDChQs3a60rzrTeUB3ITQWmApVACbBWKTVDa90GTNRa1ymlyoH3lVKfa633R2+stX4eeB6goqJCV1ZWDriQqqoqBrK9Lxjm5ufW0+7t4MOS53E2VMN1/0XRnG9QNOBqBmegbYk3idIOkLbEK2lL//XndNE6oDTqeYm5LFotsFprHdRaH8TY658KoLWuM+8PAFXA7EHWPCx+tKqaPUebeL/kBZzHN8CS52DON2JdlhBCDKn+hP5GYKpSapJSygbcCqzusc4qjL18lFIFwDnAAaWUUyllj1p+GScfC4gLb1fXs3JzDavHvUhB/f/Cdc/AzFtiXZYQQgy5M3bvaK1DSqn7gXcw+uuXa623K6WeBDZprVebr12plNoBhIEfaq2blVKXAr9WSkUw/sAsjR71Ew98wTCPr67mUedfObf5fVj0FMy5M9ZlCSHEsOhXn77Weg2wpseyx6Iea+D75i16nXXAjMGXOXxeXHeIEtdn/H3ai3D+V+DSB2JdkhBCDJukPiPXFwyz/MPdvOn4NSq7FK5fJhc3EUIktKQO/T9uqeM6/5sUWuvhK28Y4/GFECKBJW3oa61Z8fFOfmd7Ez15Eaq8MsYVCSHE8EvaK3xsP9rB3OY3ydYu1BUPxbocIYQYEUkb+qu31vKt1HcIjp8PpRfFuhwhhBgRSRn6WmuObP2AUtWAdd63Y12OEEKMmKQM/e1HO/iC931ClnQ479pYlyOEECMmKUP//R1HudryCaGpV4M9MSZpEkKI/kjK0TvHqj8kT7lhxvWxLkUIIUZU0u3pt3uDlDevJaSsMOVLsS5HCCFGVNKF/qZDLXwpZTPu4kvAnhXrcoQQYkQlXejv2L2L8pR6MqZfFetShBBixCVd6Af3rQXAWv6FGFcihBAjL6lC3x8KM759Mz5LFhRdEOtyhBBixCVV6O+pd3OR2kl74UWQYol1OUIIMeKSKvR3Hz5CeUo99rKLY12KEELERFKFfvv+TQDkTJa5doQQySmpQt9y/DMA1LhZMa5ECCFiI2lCPxzRjHHtpM02Fhx5sS5HCCFiImlCv7bVwzQO4HZOj3UpQggRM0kT+vvrGpioGkgZd2GsSxFCiJhJmtBvPlxNitLkTpTQF0Ikr6QJ/cCxHQA4xkv3jhAieSVN6Ntb9hAiFfLKY12KEELETFKEvtaaPO9BWtJKwWKNdTlCCBEzSRH6Te4Ak3UNnTlTY12KEELEVFKEfm1TGyWqEQok9IUQyS0pQr+5bh8WpUkvktAXQiS3pAh9T/0+AJzjz4lxJUIIEVtJEfrh5oMA2MfIyB0hRHJLitC3dhzBjw0yi2JdihBCxFRShH6Wt5ZWWzGkJEVzhRCiTwmfguGIZkyons6M0liXIoQQMZfwod/Y4aNUNRDMnhDrUoQQIuYSPvQbGo+TpbxYnLKnL4QQCR/67ccPA5CeL3v6QgiR8KHvaaoBILuoLLaFCCFEHEj40A+11QKQVSh7+kII0a/QV0otVkrtVkrtU0o93Mc6X1NK7VBKbVdK/T5q+V1Kqb3m7a6hKry/UlxHiaBQWcUj/dFCCBF3Us+0glLKAiwDFgG1wEal1Gqt9Y6odaYCjwCXaa1blVKF5vI84HGgAtDAZnPb1qFvSu/snmO0pTjJkymVhRCiX3v684B9WusDWusAsAK4vsc6/wAs6wpzrXWDufwq4F2tdYv52rvA4qEpvX8y/A24bHImrhBCQP9CfzxQE/W81lwW7RzgHKXUx0qpDUqpxWex7bCJRDR54UZ86RL6QggB/ejeOYv3mQpUAiXAWqXUjP5urJS6B7gHoKioiKqqqgEX4na7u7d3BzQLaKE65BjUe8ZKdFtGs0RpB0hb4pW0pf/6E/p1QPSZTSXmsmi1wCda6yBwUCm1B+OPQB3GH4Lobat6foDW+nngeYCKigpdWVnZc5V+q6qqomv7/XXHyV7nJaf0PKYP4j1jJboto1mitAOkLfFK2tJ//ene2QhMVUpNUkrZgFuB1T3WWYUZ7kqpAozungPAO8CVSimnUsoJXGkuGxHtjUcBsOVI944QQkA/9vS11iGl1P0YYW0BlmuttyulngQ2aa1XcyLcdwBh4Ida62YApdRTGH84AJ7UWrcMR0N642kxQt/hHDtSHymEEHGtX336Wus1wJoeyx6LeqyB75u3ntsuB5YPrsyB8bfXA5CRNy4WHy+EEHEnoc/IDbsaAcgukNAXQghI8NDHbZwukJJZGONChBAiPiR06Kd6m3CpTEi1xboUIYSICwkd+nZ/My5LbqzLEEKIuJHQoZ8RasFjy491GUIIETcSNvS11uSEWwnYJfSFEKJLwoa+Lxghn3ZCjjGxLkUIIeJGwoZ+q8tNtvKAoyDWpQghRNxI2NB3tRpj9C2Z0r0jhBBdEjb0PW3GGH1rpuzpCyFEl4QNfV9HEwBpORL6QgjRJWFDP+huBsCRLQdyhRCiS8KGfsgM/cw8mYJBCCG6JGzo4zVmcE7Lku4dIYToksCh30qQVLBlxLoSIYSIGwkb+qn+NlwqC5SKdSlCCBE3Ejb0bYF2Oi3ZsS5DCCHiSsKGfnqoHV+qhL4QQkRL2NDPCHcQsMm0ykIIES0hQ19rTaZ2EbI7Y12KEELElYQMfW8ghBM3kXQJfSGEiJaQod/hcmFXQUiT7h0hhIiWkKHv6TBOzEpxSOgLIUS0xAx9VysAqek5Ma5ECCHiS0KGfsBt7OnbMmRPXwghoiVm6He2A2DPlAO5QggRLSFDP+RpAyA9S0JfCCGiJWToh73Gnr4jOy/GlQghRHxJyNDXvg4AHLKnL4QQJ0nI0Fe+dsIolD0r1qUIIURcScjQTwm66MQh0yoLIUQPCRn6loALr5KLpwghRE8JGfrWkBuvRUJfCCF6SsjQt4dcBCyZsS5DCCHiTkKGflq4k4BVDuIKIURPCRn6Dt1JSEJfCCFOkXChr7UmQ3uI2CT0hRCip4QL/UBYk4UHbZfr4wohRE8JF/qhgJ9UFYE0CX0hhOipX6GvlFqslNqtlNqnlHq4l9e/qZRqVEptNW9/H/VaOGr56qEsvjehoBeAFDkbVwghTpF6phWUUhZgGbAIqAU2KqVWa6139Fj1Va31/b28hVdrPWvwpfZPxO8BwJImQzaFEKKn/uzpzwP2aa0PaK0DwArg+uEta+AiIR8AqenSvSOEED2dcU8fGA/URD2vBeb3st6NSqkFwB7ge1rrrm3SlFKbgBCwVGu9queGSql7gHsAioqKqKqq6n8LevC5jWmVD9XW0zCI94kHbrd7UD+LeJEo7QBpS7yStvRff0K/P94EXtFa+5VS9wIvAl80X5uota5TSpUD7yulPtda74/eWGv9PPA8QEVFha6srBxwISv3fQrAtJlzKJs58PeJB1VVVQzmZxEvEqUdIG2JV9KW/utP904dUBr1vMRc1k3ZixeDAAARV0lEQVRr3ay19ptPfwPMjXqtzrw/AFQBswdR75mFjAO5dodcFF0IIXrqT+hvBKYqpSYppWzArcBJo3CUUsVRT68DdprLnUopu/m4ALgM6HkAeEilmH366RnSpy+EED2dsXtHax1SSt0PvANYgOVa6+1KqSeBTVrr1cCDSqnrMPrtW4BvmpufD/xaKRXB+AOztJdRP0MqJWzs6adJ6AshxCn61aevtV4DrOmx7LGox48Aj/Sy3TpgxiBrPCsWc0/fLqEvhBCnSLgzci1hL0EsqFR7rEsRQoi4k3Chnxrx4SU91mUIIURcSrjQt0a8eJWEvhBC9CbhQt8W8eFPkdAXQojeJGToByT0hRCiVwkX+nbtI2BxxLoMIYSISwkX+mnaRyhVQl8IIXqTkKEfltAXQoheDdWEa3HDgZcGq8ylL8RICgaD1NbW4vP5YvL5OTk57Ny5MyafPdTO1Ja0tDRKSkqwWq0Dev+ECn2tNQ78YM2IdSlCJJXa2lqysrIoKytDKTXin+9yucjKSoyr5Z2uLVprmpubqa2tZdKkSQN6/4Tq3vH6AziUH2wS+kKMJJ/PR35+fkwCP5kopcjPzx/U/6gSK/Q7XcYDu4S+ECNNAn9kDPbnnFCh7/O4AVCypy+EEL1KqNAP+IzQT7HJ6B0hkklbWxvPPvvsWW93zTXX0NbWNgwVxa+ECv2g1wOARfb0hUgq7e3tvYZ+KBQ67XZr1qwhNzd3uMrqt3A4PGKflVCjd4Lmnn5qmuzpCxErP35zOzuOdgzpe04bl83jX5ne5+uPP/44+/fvZ9asWVitVtLS0nA6nezatYs9e/awZMkSampq8Pl8fPe73+Wee+4BoKysjE2bNuF2u7n66qu5/PLLWbduHePHj+eNN94gPb33KV2eeeYZnnvuOVJTU5k2bRorVqzA7XbzwAMPsGnTJpRSPP7449x444288sor/OQnP0FrzbXXXsvPfvYzADIzM7n33nt57733WLZsGenp6Xz/+9+no6ODwsJC/ud//ofi4uJeP38wEiv0/Z0AWORArhBJ5cc//jG7d+9m69atVFVVce2111JdXd09rHH58uXk5eXh9Xq56KKLuPHGG8nPzz/pPfbu3csrr7zCCy+8wNe+9jX+8Ic/cMcdd/T6eUuXLuXgwYPY7fbu7qGnnnqKnJwcPv/8cwBaW1s5evQoDz30EJs3b8bpdHLllVeyatUqlixZQmdnJ/Pnz+c//uM/CAaDXHHFFbzxxhukpaWxZs0aHn30UZYvXz7kP6uECv2Qz+jesaZJ6AsRK6fbIx8p8+bNO2kc+zPPPMOf/vQnAGpqati7d+8poT9p0iRmzZoFwNy5czl06FCf73/hhRdy++23s2TJEpYsWQLAe++9x4oVK7rXcTqdrF27lsrKSsaMGQPA7bffztq1a1myZAkWi4Ubb7wRgN27d1NdXc2iRYuIRCJorYdlLx8SLPQjAWNP35YuZ+QKkcwyMk7s+FVVVfHee++xfv16HA4HlZWVvY5zt9tPXG3PYrHg9Xr7fP8///nPrF27ljfffJN///d/7967PxtpaWlYLBbAOOlq+vTprF+/fthPNEuoA7lhv7Gnb5c9fSGSSmZmJi6Xq9fX2tvbcTqdOBwOdu3axYYNGwb1WZFIhJqaGhYuXMjPfvYz2tvbcbvdLFq0iGXLlnWv19rayrx58/jwww9pamoiHA7zyiuvcMUVV5zynueeey6NjY2sX78eMKa12L59+6Dq7EtChX4kYIS+LV1CX4hkkp+fz2WXXcYFF1zAD3/4w5NeW7x4MaFQiPPPP5+HH36Yiy++eFCfFQ6HueOOO5gxYwazZ8/mwQcfJDc3lx/96Ee0trZywQUXMHPmTD744AOKi4tZunQpCxcuZObMmcydO5frr7/+lPe02WysXLmShx56iEsvvZRZs2axbt26QdXZl4Tq3iFo7uk7pHtHiGTz+9//vtfldrudt956q9fXuvrtCwoKqK6u7l7+gx/8oM/PsVqtfPTRR6csz8zM5MUXXzxl+W233cZtt912ynK3233S81mzZrF27Vrp3jkrAeneEUKI00msPf2Ql4C2YEu1xboSIUQCuO+++/j4449PWvbd736Xu+++O0YVDV5Chb4KevFhRyJfCDEUog/MJoqE6t5RIR9+JZEvhBB9SajQt4S9+LGfeUUhhEhSCRb6PvxKQl8IIfqSUKGfGvYSkB59IYToU2KFfsRPQPb0hRBnkJlpnMtz9OhRbrrppl7XqaysZNOmTSNZ1ohIqNC3RnwEUyT0hRD9M27cOFauXBnrMs447/9QSqghm9aIn6CM0Rcitt56GOrPfgKy0xo7A65e2ufLjz/+OJMnT+a+++4D4IknniA1NZUPPviA1tZWgsEgTz/99ClTIBw6dIi/+7u/o7q6Gq/Xy9133822bds477zzTjvhWjgc5tvf/nb33Pnf+ta3+N73vse+ffv4x3/8RxobG7FYLLz++uuUl5fzL//yL7z11lsopfjRj37ELbfcQlVVFf/2b/920rz/v/vd7/jlL39JOBxm/vz5PPvss92Tsg2VhAp9u/YTkj19IZLOV7/6VR599NHu0H/ttdd45513ePDBB8nOzqapqYmLL76Y6667rs8Li//3f/83DoeDnTt38tlnnzFnzpw+P2/r1q3U1dV1T93QNaf+7bffzsMPP8wNN9yAz+cjEonwxz/+ka1bt7Jt2zaampq46KKLWLBgAQBbtmzpnvd/586dvPrqq7z77rvk5eXxne98h5dffpk777xzKH9UCRb6SOgLEXOn2SMfLjNnzqShoYGjR4/S2NiI0+lk7NixfO9732Pt2rWkpKRQV1fH8ePHGTt2bK/vsXbtWh588EHAmC//wgsv7PPzysvLOXDgAA888ADXXnstV155JS6Xi7q6Om644QbAmDoZ4KOPPuK2227DYrFQVFTEFVdcwcaNG8nOzj5p3v+//vWvbN68mcrKSlJSUvB6vRQWFg7ljwlItNDXfsIS+kIkpZtvvpmVK1dSX1/PLbfcwssvv0xjYyObN2/GarVSVlbW6zz6A+F0Otm2bRvvvPMOzz33HK+99hq/+tWvzvp9ouf911pz11138a//+q8y4Vp/hMMR0vETTpE+fSGS0S233MKKFStYuXIlN998M+3t7RQWFmK1Wvnggw84fPjwabdfsGBB90yd1dXVfPbZZ32u29TURCQS4cYbb+Tpp59my5YtZGVlUVJSwqpVqwDw+/14PB6+8IUv8OqrrxIOh2lsbGTt2rXMmzfvlPf80pe+xMqVK2lsbASgpaXljDUPRMLs6ft8XjKUJmJJi3UpQogYmD59Oi6Xi/Hjx1NcXMztt9/OV77yFWbMmEFFRQXnnXfeabf/p3/6J+6++27OP/98zj//fObOndvnunV1ddx9991EIhEAfvrTnwLw0ksvce+99/LYY49htVp5/fXXueGGG1i/fj0zZ85EKcXPf/5zxo4dy65du056z2nTpvH00093X37RarWybNkyJk6cOJgfyykSJ/S9bjJA9vSFSGLRly0sKCjovhJVT11z2ZeVlXUfjE1PTz/pGrenM3PmTLZs2XLK8qlTp/L++++fsvwXv/gFv/jFL05aVllZSWVl5UnLbrnlFq655prYd+8opRYrpXYrpfYppR7u5fVvKqUalVJbzdvfR712l1Jqr3m7ayiLj5adbqNj8t+RXzRhuD5CCCFGvTPu6SulLMAyYBFQC2xUSq3WWu/oseqrWuv7e2ybBzwOVAAa2Gxu2zok1UexZjixfuNl/FVVQ/3WQogkNn/+fPx+/0nLXnrpJWbMmBGjiganP90784B9WusDAEqpFcD1QM/Q781VwLta6xZz23eBxcArAytXCCFG1ieffBLrEoZUf7p3xgM1Uc9rzWU93aiU+kwptVIpVXqW2wohRjmtdaxLSAqD/TkP1YHcN4FXtNZ+pdS9wIvAF/u7sVLqHuAegKKiIqoG0UXjdrsHtX08SZS2JEo7QNrSl8zMTGpra8nJyenzjNfhFA6HcblcI/65w+F0bdFa097eTmdn54C/u/6Efh1QGvW8xFwWXUhz1NPfAD+P2rayx7ZVPT9Aa/088DxARUWF7nlE+2xUVVWdckR8tEqUtiRKO0Da0pdgMEhtbS11dXVnXnkY+Hy+7jNgR7sztSUtLY2ZM2ditVoH9P79Cf2NwFSl1CSMEL8V+Hr0CkqpYq31MfPpdcBO8/E7wE+UUk7z+ZXAIwOqVAgRt6xWa/d0ArFQVVXF7NmzY/b5Q2m423LG0Ndah5RS92MEuAVYrrXerpR6EtiktV4NPKiUug4IAS3AN81tW5RST2H84QB4suugrhBCiJHXrz59rfUaYE2PZY9FPX6EPvbgtdbLgeWDqFEIIcQQSZi5d4QQQpyZirdhVkqpRmAwswwVAE1DVE6sJUpbEqUdIG2JV9IWmKi1HnOmleIu9AdLKbVJa10R6zqGQqK0JVHaAdKWeCVt6T/p3hFCiCQioS+EEEkkEUP/+VgXMIQSpS2J0g6QtsQraUs/JVyfvhBCiL4l4p6+EEKIPiRM6J/pQi/xTil1SCn1uXkRmk3msjyl1LvmBWjejZrOIq4opZYrpRqUUtVRy3qtXRmeMb+nz5RSc2JX+an6aMsTSqm6qIsEXRP12iNmW3Yrpa6KTdW9U0qVKqU+UErtUEptV0p911w+qr6b07Rj1H0vSqk0pdSnSqltZlt+bC6fpJT6xKz5VaWUzVxuN5/vM18vG3QRWutRf8OYHmI/UA7YgG3AtFjXdZZtOAQU9Fj2c+Bh8/HDwM9iXWcftS8A5gDVZ6oduAZ4C1DAxcAnsa6/H215AvhBL+tOM3/X7MAk83fQEus2RNVXDMwxH2cBe8yaR9V3c5p2jLrvxfzZZpqPrcAn5s/6NeBWc/lzwD+Zj78DPGc+vhXjYlWDqiFR9vS7L/SitQ4AXRd6Ge2ux5imGvN+SQxr6ZPWei3GnEvR+qr9euD/asMGIFcpVTwylZ5ZH23py/XACq21X2t9ENiH8bsYF7TWx7TWW8zHLoyJEMczyr6b07SjL3H7vZg/W7f51GreNMZU9CvN5T2/k67vaiXwJTXIuasTJfQT4WItGviLUmqzeX0BgCJ9YvbSeqAoNqUNSF+1j9bv6n6zy2N5VDfbqGmL2S0wG2PPctR+Nz3aAaPwe1FKWZRSW4EG4F2M/4m0aa1D5irR9Xa3xXy9HcgfzOcnSugngsu11nOAq4H7lFILol/Uxv/vRuVQq9Fcu+m/gcnALOAY8B+xLefsKKUygT8A/6y17oh+bTR9N720Y1R+L1rrsNZ6Fsb1ReYB543k5ydK6J/xQi/xTmtdZ943AH/C+GU43vXfa/O+IXYVnrW+ah9135XW+rj5DzUCvMCJroK4b4tSyooRlC9rrf9oLh51301v7RjN3wuA1roN+AC4BKMrrWvW4+h6u9tivp4DNDMIiRL63Rd6MY963wqsjnFN/aaUylBKZXU9xrjYTDVGG+4yV7sLeCM2FQ5IX7WvBu40R4pcDLRHdTXEpR792jdgfDdgtOVWc4TFJGAq8OlI19cXs+/3t8BOrfV/Rr00qr6bvtoxGr8XpdQYpVSu+TgdWIRxjOID4CZztZ7fSdd3dRPwvvm/s4GL9dHsobphjDzYg9E/9mis6znL2ssxRhtsA7Z31Y/Rd/dXYC/wHpAX61r7qP8VjP9eBzH6I7/dV+0YoxeWmd/T50BFrOvvR1teMmv9zPxHWBy1/qNmW3YDV8e6/h5tuRyj6+YzYKt5u2a0fTenaceo+16AC4G/mTVXA4+Zy8sx/jDtA14H7ObyNPP5PvP18sHWIGfkCiFEEkmU7h0hhBD9IKEvhBBJREJfCCGSiIS+EEIkEQl9IYRIIhL6QgiRRCT0hRAiiUjoCyFEEvn/LUzVishULTwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize(model.score_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(303, 12) (303,)\n",
      "(242, 12) (242,)\n",
      "(61, 12) (61,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/abbo/anaconda3/envs/dev/lib/python3.6/site-packages/sklearn/preprocessing/label.py:235: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./heart.csv', index_col=0)\n",
    "\n",
    "X = df.loc[:, df.columns != 'target'].copy()\n",
    "y = df.loc[:, df.columns == 'target'].copy()\n",
    "\n",
    "X = X.values\n",
    "y = LabelEncoder().fit_transform(y)\n",
    "\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.2)\n",
    "print(X.shape, y.shape)\n",
    "print(train_X.shape, train_y.shape)\n",
    "print(valid_X.shape, valid_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyGradientBoostingClassifier:\n",
    "    \n",
    "    def __init__(self, n_estimators=100, lr=0.1, max_depth=3, verbose=False):\n",
    "        self.n_estimators = n_estimators\n",
    "        self.lr = lr\n",
    "        self.max_depth = max_depth\n",
    "        self.verbose = verbose\n",
    "        \n",
    "        self.estimator_list = None\n",
    "        self.is_first = True\n",
    "        self.F = None\n",
    "        self.score_list = list()\n",
    "        \n",
    "    def fit(self, train_X, train_y):\n",
    "        self.estimator_list = list()\n",
    "        self.F = np.zeros_like(train_y, dtype=float)\n",
    "        \n",
    "        for i in range(1, self.n_estimators + 1):\n",
    "            # get negative gradients\n",
    "            neg_grads = train_y - self.logit(self.F)\n",
    "            base = DecisionTreeRegressor(max_depth=self.max_depth)\n",
    "            base.fit(train_X, neg_grads)\n",
    "            train_preds = base.predict(train_X)\n",
    "            self.estimator_list.append(base)\n",
    "            \n",
    "            if self.is_first:\n",
    "                self.F = train_preds\n",
    "                self.is_first = False\n",
    "            else:\n",
    "                self.F += self.lr * train_preds\n",
    "                \n",
    "            train_preds = self.logit(self.F) > 0.5\n",
    "            train_score = r2_score(train_y, train_preds)\n",
    "            valid_preds = self.predict(valid_X)\n",
    "            valid_score = r2_score(valid_y, valid_preds)\n",
    "            iter_score = dict(iter=i, train_score=train_score, valid_score=valid_score)\n",
    "            self.score_list.append(iter_score)\n",
    "            if self.verbose:\n",
    "                print(iter_score)\n",
    "                \n",
    "    def predict(self, X):\n",
    "        F = np.zeros_like(len(X), dtype=float)\n",
    "        is_first = True\n",
    "        for base in self.estimator_list:\n",
    "            preds = base.predict(X)\n",
    "            if is_first:\n",
    "                F = preds\n",
    "                is_first = False\n",
    "            else:\n",
    "                F += self.lr * preds\n",
    "        return self.logit(F) > 0.5\n",
    "    \n",
    "    @staticmethod\n",
    "    def logit(F):\n",
    "        return 1.0 / (1.0 + np.exp(-F))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor train_score: 0.9834 valid_score: 0.7705\n",
      "MyGradientBoostingRegressor train_score: 0.7171 valid_score: 0.8361\n"
     ]
    }
   ],
   "source": [
    "model = GradientBoostingClassifier(n_estimators=100, max_depth=3)\n",
    "model.fit(train_X, train_y)\n",
    "print('GradientBoostingRegressor train_score: {:.4f} valid_score: {:.4f}'.format(\n",
    "    r2_score(train_y, model.predict(train_X)), accuracy_score(valid_y, model.predict(valid_X))))\n",
    "\n",
    "model = MyGradientBoostingClassifier(n_estimators=100, max_depth=3)\n",
    "model.fit(train_X, train_y)\n",
    "print('MyGradientBoostingRegressor train_score: {:.4f} valid_score: {:.4f}'.format(\n",
    "    r2_score(train_y, model.predict(train_X)), accuracy_score(valid_y, model.predict(valid_X))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8VOWd+PHPN5P7jQRyARJuAnIXkAjeqvGOVQHXdtF1t/a26FZF7ba7duuqtXa3rd3t6m9tXevS7bZVqmgBldZ6IbJaUUACBAi3AJKEkJCQkAm5z/P748wkk8lMMpnMZCaZ7/v1yos5Z55zzvPkkO8885znIsYYlFJKRYeYcGdAKaXU0NGgr5RSUUSDvlJKRREN+kopFUU06CulVBTRoK+UUlFEg75SSkURDfpKKRVFNOgrpVQUiQ13BjxlZWWZyZMnB3x8U1MTKSkpwcvQMBCNZYboLHc0lhmis9wDLfOOHTtOG2Oy+0sXcUF/8uTJbN++PeDji4qKKCwsDF6GhoFoLDNEZ7mjscwQneUeaJlF5Lg/6bR5RymloogGfaWUiiIa9JVSKopEXJu+N+3t7ZSXl9PS0tJv2lGjRrF///4hyFXkCHaZExMTyc/PJy4uLmjnVEpFhmER9MvLy0lLS2Py5MmISJ9pGxsbSUtLG6KcRYZgltkYQ21tLeXl5UyZMiUo51RKRY5h0bzT0tLCmDFj+g34avBEhDFjxvj1rUopNfwMi6APaMAfQvq7VmrkGhbNO0opNdTqz7Xxp72n+GJBfo+KUGtHJ2s+OEZzW0evY2wxMfzVkolkpyX02P/6rkoOnWrs95pjRyXxV0smDj7zfdCgr5RSXvzXljJ+XnSECaOTuWTqmK79G4or+dEfSwHw/FJsDNhb2/nuTbO79lU3tvDA2p04TO/0nhZMyNCgHynq6+t58cUX+cY3vjGg4z7/+c/z4osvkpGREaKcKaWCzeEwbNhZAcCG4gqPoF/BpDHJFH2rsFdT6Nd/tZ2Nuyp5+MZZ2GKs917fdRKHgXe+eSXTclKHrhA+DJs2/XCrr6/nZz/7Wa/9HR29v+K527RpU0QE/M7OznBnQalhY9uxOiobWshKTeDNPSdpabf+fqoaWvjzkVpWLMjz+uzr1oV5nDrbytay2q5963dWMC9vVEQEfBiGNf3vvb6XfZVnfb7f2dmJzWYb0Dlnj0/nsVvm9Jnm4Ycf5siRIyxYsIC4uDgSExPJzMyktLSUgwcPsmLFCk6cOEFLSwsPPPAAq1atArrnErLb7dx4441cfvnl/PnPfyYvL48NGzaQlJTk9XrPPPMMzz33HLGxscyePZu1a9dit9u5//772b59OyLCY489xm233cYrr7zCT3/6U4wx3HTTTfzoRz8CIDU1lbvvvpt33nmHZ599lqSkJL75zW9it9vJysrif/7nfxg3btyAfldKRYP1xZUkx9t4csVc7vnNDooOVLN07jhe31WJMbBiYZ7X466ZlUNqQizrd1Zw2bQsjtTY2VPRwCM3zRriEvimNX0//fCHP2Tq1KkUFxfz1FNP8emnn/L0009z8OBBANasWcOOHTvYvn07zzzzDLW1tb3OcejQIe6991727t1LRkYGr776ap/X27lzJ7t37+a5554D4Pvf/z6jRo1iz5497N69m6uvvprKykoee+wx3nvvPYqLi9m2bRvr168HrFn6lixZwq5du1iyZAn3338/69atY8eOHXz1q1/lu9/9bgh+U0oNb60dnby5u5LrZ+dy7awcslITWL+zEoD1xRXMzx/FlCzvs18mxtlYOncsfyypoqW9kw07K4gRWDZ//FAWoU9+1fRFZCnwNGADXjDG/NDj/Z8CVzk3k4EcY0yG8727gEec7z1pjPnVYDLcX418qAZnLV68uMfgpWeeeYbf//73AJw4cYJDhw4xZsyYHsdMmTKFBQsWALBo0SKOHTvm8/wXXHABd955JytWrGDFihUAvPPOO6xdu7YrTWZmJlu2bOHyyy8nO9uaUfXOO+9ky5YtrFixApvNxm233QbAgQMHKCkp4brrrgOsb0Ray1eqt6IDNZxt6WD5wjxibTHcMn8cv936GTuO17G38iyP3jy7z+NXLMhj3Y5y3t1fzfriSi6dmkVOeuIQ5b5//QZ9EbEBzwLXAeXANhHZaIzZ50pjjHnILf39wELn69HAY0ABYIAdzmPPBLUUYeA+z3VRURHvvPMOH330EcnJyRQWFnod3JSQ0N2Ny2az0dzc7PP8b775Jlu2bOH111/nBz/4AXv27BlwHhMTE7uauowxzJkzh48++mjA51EqmmwormBMSjyfm5YFWEH8lx8e45sv7yJG4Ob5fVeWLpk6hpy0BH78Vimf1Z3j/qunDUW2/eZP885i4LAxpswY0wasBZb3kf4O4CXn6xuAt40xdc5A/zawdDAZDpe0tDQaG733s21oaCAzM5Pk5GRKS0vZunXroK7lcDg4ceIEV111FT/60Y9oaGjAbrdz3XXX8eyzz3alO3PmDIsXL+bDDz/k9OnTdHZ28tJLL3HllVf2OueMGTOoqanpCvrt7e3s3bt3UPlUaqQ529LOO/uruWX+eGJtVni8IH8U52WlcLz2HJdPzyYnre9auy1GWDZ/PMdrz5EQG8PSuWOHIut+86d5Jw844bZdDizxllBEJgFTgPf6ONb7E5AIN2bMGC677DLmzp1LUlISubm5Xe8tXbqU5557jlmzZjFjxgwuvvjiQV2rs7OTv/7rv6ahoQFjDKtXryYjI4NHHnmEe++9l7lz52Kz2Xjsscf4i7/4Cx5//HGuuuqqrge5y5f3/kyOj49n3bp1rF69moaGBjo6OnjwwQeZM6fv5jKlhpvv/n4Pk8Yks+qKqT7T/O9Hx3iu6Eiv/W2dDto6HCxf0N0GLyIsX5DHT985yIoF/rXNr1iYxwsfHOXa2bmkJUbWxIVijOk7gcgXgKXGmK87t/8GWGKMuc9L2n8E8o0x9zu3vwUkGmOedG7/M9BsjPmJx3GrgFUAubm5i9zbrcGaRXLaNP++IgXSe2e4C0WZDx8+TENDQ1DPGWx2u53U1MjoBjdUorHM4H+5a5sd/P37zaTFwU+vSiY2pne3SocxfOv9ZuJjYHpm77+b0YnCimlxPbpk2tsMfzrezs3nxRFv63+aEmMMfzzWwfxsG+NTA+svM9B7fdVVV+0wxhT0l86fmn4FMMFtO9+5z5vbgXs9ji30OLbI8yBjzPPA8wAFBQXGc4mw/fv3+/1wVmfZDI7ExEQWLlwY1HMGmy6hFz38LffPi44ApTS2Q8z42RTOzO2V5uOyWupatvL07QtYvsD/hoebB5Bf6O7ZEqhQ3Wt/PoK2AdNFZIqIxGMF9o2eiURkJpAJuD8pfAu4XkQyRSQTuN65Tznde++9LFiwoMfPL3/5y3BnS6lhaUNxBRfkjyIzOa6rm6UnVx/862b3/kCIBv3W9I0xHSJyH1awtgFrjDF7ReQJYLsxxvUBcDuw1ri1Fxlj6kTk+1gfHABPGGPqgluE4c39waxSKnD7T56ltKqRJ5bP4eCpRtbtKMfe2kFqQneYc/XBv2HOWJLjh93Y1KDwq9TGmE3AJo99j3psP+7j2DXAmgDzp5RSfllfXEFsjHDTvHHMHpfOb7Z+xp/2VvEXF+Z3penqg+/nA9mRSEfkKqWGPYfDsLG4kivOz2ZMagKLJmWSn5nE+uKeTTwbiivISo3ncmcf/GikQV8pNex9fLSOkw0tXXPiiAgrFuTxwaEaqhutgZKuPvg3X9DdBz8aRWejllIq7BwOw9HaJqZm9+6WWHyinqbW7hls99V2Enf4tM9z/fqj46TE27huVvfD2RULx/Ofmw/z/PtlXDUzh61ltbR1OHxOlhYtNOiHSGpqKna7ncrKSlavXs26det6pSksLOQnP/kJBQX9dq1VasT53fYTfOe1Pby5+nLmjB/Vtf+jI7Xc8Qsvo9q3fdzn+b6wKJ+k+O5+99Ny0pifP4oXPjjKCx8cBWBqdgrz80f5OkVU0KAfYuPHj/ca8IdaR0cHsbF6u1XkeHVHOQC//7SiR9B/7dNyUhNieeGuAmKcA6R27tzZ57gREZg9Lr3X/jVfvogjNU1d25OzkqN+DejhFwX+8DBU+Z58LKmzA2wDLNbYeXDjD/tM8vDDDzNhwgTuvdcae/b4448TGxvL5s2bOXPmDO3t7Tz55JO9pkA4duwYN998MyUlJTQ3N/OVr3yFXbt2MXPmzD4nXOvs7ORrX/ta19z5X/3qV3nooYc4fPgw99xzDzU1NdhsNl555RWys7P59re/zR/+8AdEhEceeYSVK1dSVFTEP//zP/eY9/83v/kNzzzzDG1tbSxZsoSf/exnUTeCWYXfibpzbD9+hnhbDBt3VfKdz1srTbW0d/KHkipumDOWi8/rnqX23HEbi6eMHvB1xqQmMCY1of+EUWT4Bf0wWblyJQ8++GBX0H/55Zd56623WL16Nenp6Zw+fZqLL76YZcuW+axJ/PznPyc5OZn9+/eze/duLrzwQp/XKy4upqKigpKSEsBauQusqZMffvhhbr31VlpaWnA4HLz22msUFxeza9cuTp8+zUUXXcQVV1wBwKeffkpJSQlTpkxh//79/O53v+PDDz8kLi6Ob3zjG/z2t7/lS1/6UjB/VUr1a0OxNaj/Wzecz79sKuWjI7VcPj2Ld/dXY2/t4NYob3cPpeEX9PupkTeHaBqGhQsXUl1dTWVlJTU1NWRmZjJ27FgeeughtmzZQkxMDBUVFZw6dYqxY73PqrdlyxZWr14NWPPlX3DBBT6vd95551FWVsb999/PTTfdxPXXX09jYyMVFRXceuutgDVVAsBHH33EHXfcgc1mIzc3lyuvvJJt27aRnp7eY97/d999lx07dnDRRRcB0NzcTE5OTtB+R0r5wxjD+uJKFk8ezZcumcz/e/cw64sruHx6FuuLK8hJS+ixJq0KruEX9MPoi1/8IuvWraOqqoqVK1fy29/+lpqaGnbs2EFcXByTJ0/2Oo9+IDIzM9m1axdvvfUWzz33HC+//DJPP/30gM/jPu+/MYa77rqLf/3Xfw1KHpUKxN7KsxyutvODW+d2rTT1h5IqvnX9DIoOVHPXJZO7FhVXwRe9nVUDsHLlStauXcu6dev44he/SENDAzk5OcTFxbF582aOHz/e5/FXXHEFL774IgAlJSXs3r3bZ9rTp0/jcDi47bbbePLJJ/n0009JS0sjPz+/aznE1tZWzp07x6WXXsrvfvc7Ojs7qampYcuWLSxevLjXOa+55hrWrVtHdXU1AHV1df3mWalg21BcQZzNGjkL1jTE9tYOvvXKLto7TdR3qQw1rekPwJw5c2hsbCQvL49x48Zx5513cssttzBv3jwKCgqYOXNmn8f/3d/9HV/5yleYNWsWs2bNYtGiRT7TVlRU8JWvfAWHwwHQVTv/9a9/zd13382jjz5KXFwcr7zyCrfccgvFxcXMnz8fEeHHP/4xY8eOpbS0tMc5Z8+ezZNPPsn111+Pw+EgLi6OZ599lkmTJg3yN6OUfzodhg3FlVx5fg4ZyfEAXHyetdLUB4dPMzU7hTnje/fCUcGjQX+A3JctzMrK8rn8oN1uB2Dy5MldD2OTkpLwXCvAl/nz5/Ppp5/22j99+nTee++9HvsaGxt56qmneOqpp3rsLyws7DU168qVK1m5cqVfeVAqUDuOn+E3W4/juV6HvbWD6sbWHg9qbTHC8gXj+cX/HeXWhXlR36Uy1DToK6WC7rdbj/PG7krGZyT1em/x5NFcM6tnB4I7l0xiV3kDX1g0oVd6FVwa9CPAkiVLaG1t7bHv17/+NfPmzQtTjpQanPL6ZhZMyOCVey71K/3krBRevvuSEOdKwTAK+saYEfu17+OP+x5ePtT6W0JTqf5U1jezaFJmuLOhvBgWvXcSExOpra3VYDQEjDHU1tZ2jQFQaqA6HYaqhhbyvDTtqPAbFjX9/Px8ysvLqamp6TdtS0tL1AWsYJc5MTGR/Pz8/hMq5UV1YwsdDuO1PV+Fn19BX0SWAk9jLZf4gjGm17BYEflL4HHAALuMMX/l3N8JuLq8fGaMWTbQTMbFxXWNKu1PUVFRxC/oHWzRWGYVuSrrrTml8jI16EeifoO+iNiAZ4HrgHJgm4hsNMbsc0szHfgOcJkx5oyIuD+abzbGLAhyvpVSEar8jDPoa00/IvnTpr8YOGyMKTPGtAFrgeUeaf4WeNYYcwbAGFMd3GwqpYaLynprKhJt3olM/gT9POCE23a5c5+784HzReRDEdnqbA5ySRSR7c79KwaZX6VUhKusb2ZUUhypCcPikWHUCdZdiQWmA4VAPrBFROYZY+qBScaYChE5D3hPRPYYY464Hywiq4BVALm5uRQVFQWcEbvdPqjjh6NoLDNEZ7mHQ5l3H2khPdYENZ/DodzBFqoy+xP0KwD3YXL5zn3uyoGPjTHtwFEROYj1IbDNGFMBYIwpE5EiYCHQI+gbY54HngcoKCgwnlMHDERRUVGvqQdGumgsM0RnuYdDmX9YvIUZ+ckUFgZvGdDhUO5gC1WZ/Wne2QZMF5EpIhIP3A5s9EizHquWj4hkYTX3lIlIpogkuO2/DNiHUmrEqjjTTF5GdHWbHk76rekbYzpE5D7gLawum2uMMXtF5AlguzFmo/O960VkH9AJfNsYUysilwL/JSIOrA+YH7r3+lFKjSxnW9ppbO3Q7poRzK82fWPMJmCTx75H3V4b4JvOH/c0fwZ0AhmlooSrj7723Ilcw2IaBqXU8FBxRoN+pNOgr5QKGldNP1+DfsTSoK+U6ldJRUNXQO9LeX0z8bYYslIThiBXKhAa9JVSfWpp7+SO57fyj6/6XtPZpbK+hXEZicTowuYRS4O+UqpP7+w/RWNrBx8ePk11Y0ufaSvrmxk/Spt2IpkGfaVUn9bvrCQtIRaHgdd3newzbcWZZu2uGeE06CulfDrT1Mb7B6u5ffEE5uals6HYczB+t/ZOB6caW7TnToTToK+U8unNPSdp7zQsX5DHigV57C5voKzG7jVtVUMLxqCjcSOcBn2llE8biiuYlpPKnPHp3DJ/PCKwvrjSa9oK1+IpGclDmUU1QBr0lVJenag7x7ZjZ7h1YR4iQm56IpdOHcOG4gqv61V3j8bVmn4k06CvlPJq4y6rRr9s/viufSsW5HG89hw7T9T3Sq+jcYcHXeVAKdWLMYb1OysomJTJhNHdzTVL547lkfUl3P/iTnLTew7AOnGmmazUeBLjbEOdXTUAGvSVUr3sO3mWQ9V2vr9ibo/9aYlxfPuGGbx/sKbXMTPHpnHZtKyhyqIKkAZ9pVQvG4oriY0Rbpo3rtd7X//ceXz9c+eFIVcqGLRNXynVQ6fDsKG4gsIZ2YxOiQ93dlSQadBXSvXwcVktp862snxBXrizokJAg75Sqof1xRWkxNu4dlZuuLOiQsCvoC8iS0XkgIgcFpGHfaT5SxHZJyJ7ReRFt/13icgh589dwcq4Uir4Wto7+cOeKpbOHUdSvPbCGYn6fZArIjbgWeA6oBzYJiIb3de6FZHpwHeAy4wxZ0Qkx7l/NPAYUAAYYIfz2DPBL4pSarDeK62msbWDFQvH959YDUv+9N5ZDBw2xpQBiMhaYDngvsD53wLPuoK5Mabauf8G4G1jTJ3z2LeBpcBLwcm+UiPciW1Q+kbXZlZ9IlAY1EvUNLZytqUdgFe2nyA7LYFLp2rXy5HKn6CfB5xw2y4HlnikOR9ARD4EbMDjxpg/+jhWnw4p5a+if4Ejm8EWD452zo9Nx/pSHRwV9c1c9VQRbZ2Orn1fu3wKNl0EZcQKVj/9WGA6VhUkH9giIvP8PVhEVgGrAHJzcykqKgo4I3a7fVDHD0fRWGaIjnIXVJXRMqaAknmPMPnob5l0/BWKNr8LEpz29jfK2mjrdPDlOfEk2oQYgXmJpygqqu7/4CEUDffaU6jK7E/QrwAmuG3nO/e5Kwc+Nsa0A0dF5CDWh0AFPb+L5gNFnhcwxjwPPA9QUFBgCgsLPZP4raioiMEcPxxFY5khSsq97Rypk2ZZ5Uw6CMdfpvCieZCaM+hTG2P4l51bWDQplcf/5tLB5zWEouJeewhVmf3pvbMNmC4iU0QkHrgd2OiRZj3O4C4iWVjNPWXAW8D1IpIpIpnA9c59Sqn+OBxw7jSkOAN8arb1b1PvKRACsf9kIwdP2VmxUFtco0m/Qd8Y0wHchxWs9wMvG2P2isgTIrLMmewtoFZE9gGbgW8bY2qdD3C/j/XBsQ14wvVQVynVj+Y6MI7uWr0r+NuD0/SyobjC51QLauTyq03fGLMJ2OSx71G31wb4pvPH89g1wJrBZVOpKOQK7inOGr4r+Aehpm9NtVDJlefrVAvRRkfkKhWpmjyCfoqzG2UQavofH62l6mwLy7VpJ+po0FcqUtmdNXpXDT8xA4fEdn8YDMKGnZWkxNu4TqdaiDo6tbJSAbK3dvCLLWW0djj6TxyAi06WcA3wH1vraYktBeBuGUX5gcO82VE6qHNv2nOSG+aO1akWopAGfaUCtPaTz3j63UPEx4bmC/PomDI+JzZ+9nEtYPV/uCkmnTPV5aypOjqocyfYYrhzycQg5FINNxr0lQrQhuJK5ual88b9nwvNBda/AUdyOfj3n+/aVfvMU8xL6ODg3TeG5ppqxNM2faUCcKTGzp6KBlaEcs75puruvvlObfEZ3W39SgVAg75SAdiwswIRuGV+CGejtFd39813aovPsLpsGhO666oRTYO+UgNkjGF9cSWXTc0iNz0xdBdqqunurunUHjcKHO3QrLOTq8Bo0FdqgD79rJ7P6s6xfEEIa/nGWEHfW/MOBG0qBhV9NOgrNUAbiitIiI1h6dyxobtISz10tnlv3oGgTcWgoo8GfaUGoL3TwRu7T3Lt7FzSEuNCd6Gm09a/HrNptsdpTV8NjgZ9NSxUn21h2X9+wMFTjT32d3Q6uP35j3h3/6mAzvvzoiN8+5VdfabZXFrN4h+8Q8GTb7P4B+9Q19QW2l470HveHSdt3lGDpf301bDw6qcV7C5v4P0DNZyfm9a1/+jpJraW1WEMXDPAKQXaOhz815Yj1J9rZ/U105kwOtlrujUfHsVh4IY5VnNOZnI8hTOyvaYNGtdUC71q+mnWAiravKMCpEFfDQsbiq11e0qretb0XdsfH62jsr6Z8RlJfp9zy8Ea6s9Za8Nu3FXJvVdN65WmurGFDw+f5huF0/jWDTMCzf7Aufrie7TpIzHWxGtBmH9HRSdt3lERb//Js5RWNWKLEUqrzvZ4r7TqLK7lXDfuqhzQedcXV5CZHMfCiRn8fmcFxkvf9zd2ncRhYMXCEPbU8aap2grwyaN7v5eSrQO0VMA06KuIt764AluMcOvCPA5V2+lwW8S79GQj03JSWTgxg/U7PVfx9K2xpZ23953i5gvGc9uF+RyutrO38myvdOuLK5ibl860nDQvZwkhezUkj4EYLxOipWRrTV8FTIO+imgOh+F152Ifl04dQ1uHg2O1TV3vl1Y1MnNsOisW5FFa1djrm4Avb+09RWuHgxULx3PTvHHExkhXE5JLWY2d3eUhnmrBl6aa3k07Lqk5WtNXAfMr6IvIUhE5ICKHReRhL+9/WURqRKTY+fN1t/c63fZ7rq2rVJ8+OVZHZUMLyxeMZ+bYdMBa2xXgXLuhor6ZmePSuPmCcdhihPU7/Wvi2VBcwYTRSVw4MZPMlHgKZ+SwcVclnY7uJp71xZWhn2rBFy8Ds7qkZOtUDCpg/QZ9EbEBzwI3ArOBO0RktpekvzPGLHD+vOC2v9lt/zIvxynl0/qdFaTE27h+9lim5qT0aNcvt1vNPDPHpjEmNYErpmexsbgCh6PvYFh91no4u2JBHiLWA4EVC8dz6mwrH5fVAtZUCxuKK7h06pjQTrXgi5d5d7qk5kBHM7TZhzZPakTwp/fOYuCwMaYMQETWAsuBfaHMmBpeKuubOeDWs2Z6bir5md67QPqrtaPTWuxjTvdiH1OzU7quU97oCvrWN4AVC/N4YG0xv/roGJPHpPg87/sHa3AYWO7WbHPtrFxSE2JZ8+ExWjscVNQ3c7z2nNcePUOiqaZXd80u7gukJwzxswY17PkT9POAE27b5cASL+luE5ErgIPAQ8YY1zGJIrId6AB+aIxZP5gMq8hjjOGuNZ9wqLq75nleVgrv/v2VXTXpQGwureFsS0ePdVxnjk1nx3FrsrETjQ7SE2MZN8qqiV83O5e0xFi+93r/9ZH5EzKYlpPatZ0YZ+PmC8axdtsJ3nEO9EqJt4V2qgVfWu3Qfq57TVxPrgFbTTUwZurQ5UuNCMHqp/868JIxplVE7gZ+BVztfG+SMaZCRM4D3hORPcaYI+4Hi8gqYBVAbm4uRUVFAWfEbrcP6vjhKNxlPtbQyaHqFlZMi+OCLBsltZ28dqiJNRveY2pG4Mvx/WJnC+nxQkd5CUWV1odH/Lk2Kurb2fT2Zo7XtzM2ycb777/fdcz3Lo6jvqX//9Y5yW29fmdXZxjOvyQRnK1DoxKET7d+GHD+A5XYfJKLgdITdVR55NFut7O9tJoCoOTjzZwuaxny/IVDuP+Ph0PIymyM6fMHuAR4y237O8B3+khvAxp8vPc/wBf6ut6iRYvMYGzevHlQxw9H4S7zE6/vNdP+6U1zpqnVGGNMQ3Obmf7dTeaxDSUBn7P+XJuZ/k+bzOMbe57j3f1VZtI/vmE+OVprZvzTG+af1+8ZVN4j0vGtxjyWbszBP/V6a/PmzcY0VFrvf/KLoc9bmIT7/3g4DLTMwHbTTzw3xvjVe2cbMF1EpohIPHA70KMXjoiMc9tcBux37s8UkQTn6yzgMvRZwIjS6TC8vquSwhk5ZCTHA5CeGMe1s3J4fVcl7Z2BLRr+x5KTtHU6enWXnOFsv393fzUtnTBj7Ahs027yPu9OF1ezj3bbVAHoN+gbYzqA+4C3sIL5y8aYvSLyhIi4euOsFpG9IrILWA182bl/FrDduX8zVpu+Bv0R5KMjtVQ3tnLrwp7BecWCPGqb2vjg8OmAzvv7nRWcl5XCBfmjeuwfPyqRtMRYNjr71Lse4o4orsmN+gZdAAAevklEQVTUfD3ItcVB0middE0FxK82fWPMJmCTx75H3V5/B6vZx/O4PwPzBplHFcHWF1eQlhDL1TN7BqjCGTmMSopjw84KrprhI3j5cLKhmY+P1vHgNef3ehAsIswam84nx+qAEVrT75p3p49J3VJzdFSuCoiOyFUBa2nv5I8lVSydO5bEuJ4PbONjY/j8vHH8ad8pzrV1DOi8G4srMQafK1O5An12kpCaMALnDGyqhqRMq0bvi86/owI0Av9i1FB5Z/8p7K0drFjofZqCFQvG89Inn/H2vlM9+sT3Z31xJQsnZjA5y3tf+5njrKCfn+alzuLotCYq8+wq2tEG5wJrahpy9Sf6ruWD9X7FDjg7gEnmYhO9T+DWcjZ4A72Sx0BsQs99xoBxeJ9HyJ2js/80atA06KsBeXbzYfY5JyYrqWwgNz2Bi88b4zXtRZNHk5eRxL/96SB/2uvfIiftnQ72nzzL95bN8ZnG1Y4/wVvQ/88CuOhv4ZJv9Nz/4l9C2Wa/8hARplzZ9/vp42Hva/DvswZwUoF7PoCxc7t3tTTAv820xgUEw8RL4Kt/7Llvxy9hy0/gwRKI8dG4cHI3vHAN3PsxjD4vOHlRXmnQV347UXeOp946wNj0RFITY4mzxfD1z52HLcb7AKyYGOG+q6fx3x8c5YDHild9uXBiBsv6mO9mzvh0rp2Vy6LMhp5vtJ2DujI4VdL7oJoDMPFSmL/S73yE1aTL+n7/sgche4ZVg/ZH4yko+heoPdQz6NefsAL+RV+HsYN8/FbyKlTt6b2/qgTOVljr/nr7pgFQvc9aE7jmgAb9ENOgr/zmmoXylXsu8bnKlKc7Fk/kjsUTg5qPxDgbL9xV0HvgiuvBpueqUsZYPV3mr4RFXw5qXsImNRsu/JL/6e3VVtD3fA7g+p3NvQ0mXTq4PDVWwdEt0Nne83mE+33xFfTtPu6dCjp9kKv8Yozh9zsruGhypt8Bf8i5Appnr5bmM+Bo9z2BWTRIHmM96/D83fhaoSsQ7tNDeLtGX72NXO9pj6SQ06Cv/LK38ixHapp8PrSNCF01Ss/abD/93qNBjM0K/J416a61eIOw5m+q20Rw3q7RVy3edc+0R1LIadBXflm/s4I4m3DTvHH9Jw4XV1DxnGve3s8I12iRkuOlFl4NtgRICMIgN9e3BZ81/T4CepMfaVRQaNBX/ep0GDbuquTK87unWohITc4umY5266Fh136t6QNWbd4zqDadtn4vg5gNtcf5oec12puhrbH3fk9N1f2nUUGhQV/1y9dUCxHHvT3YvZmgKYjt1sNZSo73ppdgfQNK8dK84x7E/Wre0Tb9UNOgr/q1vriC1IRYrpkV4UGzR7Cp7rlfbNYo12iW6qN5J1hBPyEV4pI9Ar2XD19PDodb844G/VDToK/61NdUCxGnqaY7sHt+AKRk+R4YFC1Ssq0++a1uo2/7Wos30Gt4+/BNyvRdi28+A6bTStN8xuryqUImyv8KVH/e3V+NvbUj8pt2wAoquc6BR561zWhv2gG3LpXO4OuqYQfzd5OS3ftbFlj3xVdNv8ktDWi7fohp0Fd9+v3OCnLSfE+1EFGaqq1RqhLTu7YZzNrscNXVpdIZVFvqwdER3AfcqTkeTTrO+5Az27onxsui9V0fDHN6bquQ0KCvfKo/18b7B6tZNn+8z6kWIkZHqzWPTGouJGf1fqirNf3eNf1QdGXtVdOvsbqDjsqHzlZoPdv7GFfNPme2c3uYTIw3TGnQVz69ueck7Z0msgdkuXT10Ml2PrB0Bg7XFAxa0++u0Xv2iQ92Tf9crTVjpusarnsC3gdfufLR1byjNf1Q0qCvfNqws5JpOanMGT8MVqdy1VpTc3o+TGyzQ0ez1vShu0bvOS1CUNv0c6xJ4M7VOa9R031P3K/pzl4NMbGQNb17W4WMX0FfRJaKyAEROSwiD3t5/8siUiMixc6fr7u9d5eIHHL+3BXMzKvQKT9zjk+O1bFiwfheq1dFJFfNPiWn56pSOhq3my3O6iHjOV1FMH83qV6akNxr+t4e0rrGCiSkQWySPsgNsX6DvojYgGeBG4HZwB0iMttL0t8ZYxY4f15wHjsaeAxYAiwGHhORKO8sPTxsKLYW5xjI4idh5T6HjGtVKVfTjmu/6jlAqykE4xc8B2g1VTtr+j7m5QHnM5dsa1RwarbW9EPMn6mVFwOHjTFlACKyFlgO+LPA+Q3A28aYOuexbwNLgZcCy2702ld5lm3OdWE9HTrezvE/Hwvq9V7ZfoKCSRE8o6Ynu1tTRWqO1aTTZu+5X/UcoOWqhQdz/IJ7jb6z3ep3n5JjTfaG+K7pu45L0bV/Q82foJ8HnHDbLsequXu6TUSuAA4CDxljTvg4dphUHSOHMYZ7X/yUo6ebfCfavzfo1733qmlBP2fINNVAfCrEJ/esVTa5tfUra5Dayd3W61A84E7Jsv61V/f8lmWL9T7LJ1hNc9nOFcBSc6D+s+DmSfUQrEVUXgdeMsa0isjdwK+Aq/09WERWAasAcnNzey+OMQB2u31Qx0eisvpOjp5u4a9nxbNkXO9b1tTUREqK9/VkAxUjkGI/QlHRkaCeN5jc7/WsshLSY1L5uKiIzLqTzAc+/eBPjK4rZhLClm17MTGlYc1vMAz2//e0+jbGNpzkg6IiLjx5hI7YVHYH8+/FGK6QWMr376C6NokCoORoNaftRRRICs3H97HX/XrGcMXZKsrrWigrKuL8sx1k1ZXzZ488jcS/6/6Eqsz+BP0KYILbdr5zXxdjTK3b5gvAj92OLfQ4tsjzAsaY54HnAQoKCkxhYaFnEr8VFRUxmOMjUdHGvcTHfsY/rCwkPTGu9/sjsMz+6FHu4/8G8ROt7ZOZsPt7XDg9D8oOQs1orrz6mnBmNWgGfa9jtkHFmxRedjHsbIGJBcH/v7Mzl4mjE5k4cyLsgLlLroKJS+D4JFLbW3per6UB3m9n4qwLmXhpIXT+H1S9Q+EVn+uxSHo0/h8PVZn9aczbBkwXkSkiEg/cDmx0TyAi7pOsLwP2O1+/BVwvIpnOB7jXO/cpP3V0OnhjdyXXzMzxGvCVk72mZ7swWM0LwZ5mYLjr+t1Ud/ehD/o1srt/99DdhOStvd5z5a5Ujy6fKuj6rekbYzpE5D6sYG0D1hhj9orIE8B2Y8xGYLWILAM6gDrgy85j60Tk+1gfHABPuB7qKv98cPg0p+1tw6cXTbg0VcPEi63Xrnblphpnz5Cs8OUr0rg+GOuOQkdLaIJ+ag7YT/V+iO45RQP0XrnLvT+/9rgKCb/a9I0xm4BNHvsedXv9HeA7Po5dA6wZRB6j2obiStITY7lqpv4B+NTZYdUMXQHNFgdJo7sf5I5fGN78RRJXAD7lfPAfigfcKTlQVWJ96MYlW1Mug3OWzyZoa4J45zMobx8Mrv2uuXhUUOmI3Ah2rq2Dt/ZWcdMF40iIjfBpjcPpXC1getZaXQO0dN6dnly1Z1fQD0lN39m8Yz/V+55Az26bnlNB+FpyUQWNBv0I9va+U5xr69Smnf5465aZkm11/Wtr1GYCd101/RLr31DV9B3tcPqQxz3xMv9OU401K2qycxZXb0suqqAKVpdN5UXDuXbu+c0OGlsDWxTiZH0L40clsnjy6CDnbITxNgArNQfKt/XeH+3iEq1ZL2uc3VdD8btxBfqaUpjq1mvKc4oGsO5d8pjunjqJGRATp6NyQ0iDfghtO1bHR2W1LJkymtSEgf+qc9MSWbEwj5hIn9Y43Nxn2HRJybYeVIIOzPKUkgV1ZYB017CDfX5wPih2e4jeNeGbxxq67vdNpLv3jwoJDfohVFplzR3+31++KKCgr/xk9+gBAh4fANp7p4eUHCvoJ4+2RsqG4vwunk1u0HuxdM/nCjr/Tkhpm34IlVY1MmF0kgb8UGuqBluC1Wzh4q0tWVnc+82H5Pw+fvexCZA4ysuqZh750Pl3QkqDfgiVVjUyc+wwmIt+uHMNzHKfAjrFSw1TWbq6R4bo95I02pq909s1PAO6t95V3vrzq6DRoB8iLe2dHD3dxKyxaeHOysjnbWSpK9gkjLIeXqpunt0jgy0mprtJzVtAd6190NZk9dvv9cHgbNP3tp6uGjQN+iFyuNpOp8MwQ2v6oeeriQC0u6Y3rg/IUD7g9hxs5X7trrn2PaZgcEl1dvlsqQ9d/qKYBv0QKa1qBGDmOK3ph5zdS00/JcTt1sNZV00/hB+IntMquF/bc+UuXx/Y2sQTEiPnCWNLA6z/BnNO10DVLwI7x/lL4cK/CUp2DlSdJSE2hsljgjvlcdRqb4Y3v9Wj9td1r73V9OMSraYdren35qsWHuxr2OKtB7ee+1sa4KW/8t7VFrrv2RsPWqt62eJJTF0aurxGmZET9B2dcOYYSc12ONPHYiO+NJyA2sNBC/qlVY2cn5uGTfvYB8fJ3VD8G8icbC2WAt33Onduz0FALhd9FcbNH9p8Dge5c2DGTTD5c6G7xuzl1hgAz/WVp14FpW9A/XFre8oV3Quiu4y9ACZeYn04nKuDmv2Mnp4DrAxdfqPIyAn6yaPh7z5ke6BzUL/xTdj7+6BlZ//JRq6aobXMoHE1Cfzl/3YF8n7v9bWPhzpXw1NCKtzxYmivMfPz1o+n/AK4+/2+j00eDV/9o/W6swO+n0V8m7bvB4u26buk5kBznbWu5yCdtrdy2t7KDO25Ezy61m10ssVC8mji2jXoB4sGfZeuOdhPD/pUB5wPcWeN0547QdPV/quja6NOSrbW9INIg76L+4pCg7T/pDX9gtb0g8he7Xyop6uHRR0N+kGlQd8lNXjdxA5UNZKVmkBWasKgz6Wcmqq1aSdapeZo0A8iv4K+iCwVkQMiclhEHu4j3W0iYkSkwLk9WUSaRaTY+fNcsDIedN4mgwpQaVUjs7R/fnA1ndbZMqNVSg5x7Q3hzsWI0W/QFxEb8CxwIzAbuENEZntJlwY8AHzs8dYRY8wC5889QchzaKQGp3mn02E4eKqRGbka9IPK22yMKjqkZhPb2WyN1VCD5k+XzcXAYWNMGYCIrAWWA/s80n0f+BHw7aDmcKjEp0JsEtirKT5Rz8biyoBOc66tg9YOBzP1IW5wNdVoTT9adY3QrYbMSeHNywjgT9DPA064bZcDS9wTiMiFwARjzJsi4hn0p4jITuAs8Igx5v8Gk+GQcVu84Zl3D1F0oJqU+MCGMeSmJ7Bkiq52FTTtLdB6VnvuRCv3plcN+oM26MFZIhID/DvwZS9vnwQmGmNqRWQRsF5E5hhjznqcYxWwCiA3N5eioqKA82O32wM+/kJHAh0nDlB8poYlY23cPT/wB7FHdn/CkYCPHpjBlHk4SGip5hLgQPkZTrqVc6SX25toLHPa2RMsAvZsfZfaLHu4szNkQnWv/Qn6FcAEt+185z6XNGAuUCTWkOuxwEYRWWaM2Q60AhhjdojIEeB8YLv7BYwxzwPPAxQUFJiARtQ6FQU6IhegchqdZz6jrsVwxfzpFBZODTgfQ2lQZR4OynfAVpix6HPMmFHYtXvEl9uLaCwz9VPh028xb0ouLCoMd26GTKjutT+9d7YB00VkiojEA7cDG11vGmMajDFZxpjJxpjJwFZgmTFmu4hkOx8EIyLnAdOBsqCXIlhSs+lsPAXo7JgRpUlH40a1rrV1ddbNYOi3pm+M6RCR+4C3ABuwxhizV0SeALYbYzb2cfgVwBMi0g44gHuMMXXByHhIpOQQ21KH4GCWzoMfOVzdaHXGzOgUl0iHLYVYXSw9KPxq0zfGbAI2eex71EfaQrfXrwKvDiJ/Qys1hxgcTEpsITddB1ZFDJ13J+q1xY8iVtfNDQodkevO+TWyIKsD8ZwSVoVPU4216Lkuexi12uIztHknSDTou3EkW10C52W0hTknqgd7tXbXjHLtcaOCMi+W0qDfQ1Wn1Y4/PeVcmHOiemiq0aadKGfV9DXoB4MGfTeldqv5YGJC9PQFHhbs1foQN8q1xWdYS2V26LfwwdKg76akNoY2YyPH1hjurCh3WtOPem3xGdaLc4Nf7yLaadB3U3qqkfqYDOKba8OdFeXS2W6taKbz7kS19jhn0NcmnkHToO+mtKqR5rjR+sAokrhWMtMZNqNaV01f++oPmgZ9p+a2To6dbsKRkq21iUji+gDWmn5U6wr6+rc5aBr0nQ5VN+IwEJeeo7WJSOLqm601/ajWHjfKeqHfwgdNg75TqXMx89Qx462gb0yYc6QAt3l3NOhHs87YJIhL1gFaQTDoqZUjRWtHJ0UHaig51UHr3qoBH//u/lMkxsWQPmY8dLZZ3cOSMkOQUzUgdm3eUU4p2VrTD4IRE/TtLR3c/esd1sbOHQGdY/GU0cSk5VobTac16EeCphprRbP41HDnRIVbqja9BsOICfqjkuJ4c/XlbN++nYKCgoDOMWF0MlS0WBv2asiaHsQcqoA01VgDs3QuJJWSA2eOhTsXw96ICfqxthjmjB9FTbqNOeNHBX6iIC2QroLEXq0Ds5QlNRvKPwl3LoY9fZDrSRdsiCxNNfoQV1lSsuFcLTg6w52TYU2DvqfkMSAxWtOPFDrvjnJJyQHjsAK/CpgGfU8xNivw6yCQ8HN0WnOtaPOOgu4Pf/3bHBS/gr6ILBWRAyJyWEQe7iPdbSJiRKTAbd93nMcdEJEbgpHpkEvRXgIR4VydVbPT7poKuj/89Vv4oPT7INe5sPmzwHVAObBNRDYaY/Z5pEsDHgA+dts3G2sh9TnAeOAdETnfGBPZjXKp2VB3FMreD3dO/JJxZjeUjcDeLQ0nrH+1TV9B94f/sQ9BbOHNS3wK5C3q3avs1D7vFUYRK318ytDkrw/+9N5ZDBw2xpQBiMhaYDmwzyPd94EfAd9227ccWGuMaQWOishh5/k+GmzGQypjEpQVwf8uC3dO/LIAYFe4cxFCmZPCnQMVCdLGQkwc/N9PrJ9w++pbMPHi7u1zdfDcZda3U28ufwiufXwoctYnf4J+HnDCbbscWOKeQEQuBCYYY94UkW97HLvV49i8APM6dG74AVywMty58NvO4mIWLlgQ7myERnwKjJsf7lyoSJCQBt/YCvZT4c3H2Up47etQ/1nPoN9QbgX8qx+BiZf2POa1v7XSR4BB99MXkRjg34EvD+Icq4BVALm5uRQVFQWcH7vdPqjjhyN77CQajrWHOxshUg8HvTezReW9jsIyQ2SVO7Y9icuBw7s+oryu+3lTZt1O5gM761JocPT8e1xIGo7yQ+waQBlCVWZ/gn4FMMFtO9+5zyUNmAsUidW+NRbYKCLL/DgWAGPM88DzAAUFBaawsND/EngoKipiMMcPR9FYZojOckdjmSHCym0MbI1n2th0prnnaVcV7IaFn7sBxkzteUzVNKg9MqAyhKrM/vTe2QZMF5EpIhKP9WB2o+tNY0yDMSbLGDPZGDMZqzlnmTFmuzPd7SKSICJTgOmADqlTSg1fIs7J3zwe2Nr7mBE2NSdieh31W9M3xnSIyH3AW4ANWGOM2SsiTwDbjTEb+zh2r4i8jPXQtwO4N+J77iilVH9SsnqPF2iqBluC9eyhV/ps60FvZwfYwjv7jV9XN8ZsAjZ57HvUR9pCj+0fAD8IMH9KKRV5UrzU3O01Vo3e2+SAKdmAsQYbpo0dkiz6oiNylVJqoFJzes/P1VTte0yJa4xBBIwm1qCvlFID5WrTd19hz1XT95o+ckYTa9BXSqmBSs0BR7u1wp5LXzPCdtX0wz+9iwZ9pZQaqBSPIO5wOBf88VXTd34YRMCcXhr0lVJqoFwzfrqaa5rPgOn0PSNsQhrEJmrzjlJKDUspHg9mXcHc19oPItYx2ryjlFLDUNeyqs4g3jUwq49pwFOztaavlFLDUlKmtcJeV03fGfz7mgY8JVtr+kopNSzF2CA5q7vm7gr+fS34k6I1faWUGr7cB2g1VUNMLCRm9J2+6bTV0yeMNOgrpVQg3Cddc/XRj+kjpKbkWD18muuGJn8+aNBXSqlAuM+cae9jYFZX+sjoq69BXymlAuF6MGuMFfz7as+H3t08w0SDvlJKBSI1Bzqaoc3urOn3E/Q9u3mGiQZ9pZQKhHvNvana98CsrvTZ3enDSIO+UkoFwhXEa49AZ1v/bfqJGVYPnzB329Sgr5RSgXDV7E+VWP/217wTExMRA7T8CvoislREDojIYRF52Mv794jIHhEpFpEPRGS2c/9kEWl27i8WkeeCXQCllAoLV5A/tdf6t7/mHYiIAVr9LpcoIjbgWeA6oBzYJiIbjTH73JK9aIx5zpl+GfDvwFLne0eMMQuCm22llAqzlCzr32pnKOyvpg/OAV2R37yzGDhsjCkzxrQBa4Hl7gmMMWfdNlMAg1JKjWS2OEgaDacPWtv9ddkE59q64W3e8Wdh9DzghNt2ObDEM5GI3At8E4gHrnZ7a4qI7ATOAo8YY/7Py7GrgFUAubm5FBUV+Zv/Xux2+6COH46iscwQneWOxjJD5Jb7IkkhxVGHIYb3t+0BsfWZ/ry6FvIbT7Fl82bvC6i7CVmZjTF9/gBfAF5w2/4b4D/7SP9XwK+crxOAMc7Xi7A+PNL7ut6iRYvMYGzevHlQxw9H0VhmY6Kz3NFYZmMiuNy/vMmYx9KN+fFU/9J/+P+s9OfO9Jt0oGUGtpt+4rkxxq/mnQpggtt2vnOfL2uBFc4PlFZjTK3z9Q7gCHC+X59GSikV6VxNOv6057unD2MTjz9BfxswXUSmiEg8cDuw0T2BiEx327wJOOTcn+18EIyInAdMB8qCkXGllAo7V99810PdftM704XxYW6/bfrGmA4RuQ94C7ABa4wxe0XkCayvExuB+0TkWqAdOAPc5Tz8CuAJEWkHHMA9xpjwTjGnlFLB4gr6/jzEhe5vBGHstunPg1yMMZuATR77HnV7/YCP414FXh1MBpVSKmIF2rwTxgFaOiJXKaUC5Qr2/gzMAkgeYy2zGOk1faWUUl64gr2/Nf0YmxX4P/kF7H+99/u5c+ALa4KXPy806CulVKDGXgCXPQDn3+D/MVf8Axz/wPt7GZOCk68+aNBXSqlA2eLguicGdsySVdZPmGibvlJKRREN+kopFUU06CulVBTRoK+UUlFEg75SSkURDfpKKRVFNOgrpVQU0aCvlFJRRKy59yOHiNQAxwdxiizgdJCyM1xEY5khOssdjWWG6Cz3QMs8yRjT7yRAERf0B0tEthtjCsKdj6EUjWWG6Cx3NJYZorPcoSqzNu8opVQU0aCvlFJRZCQG/efDnYEwiMYyQ3SWOxrLDNFZ7pCUecS16SullPJtJNb0lVJK+TBigr6ILBWRAyJyWEQeDnd+QkVEJojIZhHZJyJ7ReQB5/7RIvK2iBxy/psZ7rwGm4jYRGSniLzh3J4iIh877/nvRCQ+3HkMNhHJEJF1IlIqIvtF5JKRfq9F5CHn/+0SEXlJRBJH4r0WkTUiUi0iJW77vN5bsTzjLP9uEbkw0OuOiKAvIjbgWeBGYDZwh4jMDm+uQqYD+HtjzGzgYuBeZ1kfBt41xkwH3nVujzQPAPvdtn8E/NQYMw04A3wtLLkKraeBPxpjZgLzsco/Yu+1iOQBq4ECY8xcwAbczsi81/8DLPXY5+ve3ghMd/6sAn4e6EVHRNAHFgOHjTFlxpg2YC2wPMx5CgljzEljzKfO141YQSAPq7y/cib7FbAiPDkMDRHJB24CXnBuC3A1sM6ZZCSWeRRwBfDfAMaYNmNMPSP8XmOt6JckIrFAMnCSEXivjTFbgDqP3b7u7XLgf41lK5AhIuMCue5ICfp5wAm37XLnvhFNRCYDC4GPgVxjzEnnW1VAbpiyFSr/AfwD4HBujwHqjTEdzu2ReM+nADXAL53NWi+ISAoj+F4bYyqAnwCfYQX7BmAHI/9eu/i6t0GLcSMl6EcdEUkFXgUeNMacdX/PWF2yRky3LBG5Gag2xuwId16GWCxwIfBzY8xCoAmPppwReK8zsWq1U4DxQAq9m0CiQqju7UgJ+hXABLftfOe+EUlE4rAC/m+NMa85d59yfd1z/lsdrvyFwGXAMhE5htV0dzVWW3eGswkARuY9LwfKjTEfO7fXYX0IjOR7fS1w1BhTY4xpB17Duv8j/V67+Lq3QYtxIyXobwOmO5/wx2M9+NkY5jyFhLMt+7+B/caYf3d7ayNwl/P1XcCGoc5bqBhjvmOMyTfGTMa6t+8ZY+4ENgNfcCYbUWUGMMZUASdEZIZz1zXAPkbwvcZq1rlYRJKd/9ddZR7R99qNr3u7EfiSsxfPxUCDWzPQwBhjRsQP8HngIHAE+G648xPCcl6O9ZVvN1Ds/Pk8Vhv3u8Ah4B1gdLjzGqLyFwJvOF+fB3wCHAZeARLCnb8QlHcBsN15v9cDmSP9XgPfA0qBEuDXQMJIvNfAS1jPLdqxvtV9zde9BQSrh+IRYA9W76aArqsjcpVSKoqMlOYdpZRSftCgr5RSUUSDvlJKRREN+kopFUU06CulVBTRoK+UUlFEg75SSkURDfpKKRVF/j8aJXC4MlwBxQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize(model.score_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
